#!/usr/bin/env python
# encoding: utf-8
"""
@author: youfeng
@email: youfeng243@163.com
@license: Apache Licence 
@file: task_base_worker.py
@time: 2016/12/7 16:17
"""
from common import util
from common.mongo import MongDb
from common.util import get_pid_log_name
from config.conf import mongo_db_target, mongo_db_source
from logger import Gsxtlogger


# data = {_id: xxx, in_time: xxx, company_name:xxx, province: xxx, base_info:{} }

class ParseObject(object):
    def __init__(self, **kwargs):
        logfile = kwargs.get('logfile', '')
        self.province = kwargs.get('province', '')

        self.target_table = kwargs.get('target_table', '')
        self.source_table = kwargs.get('source_table', '')
        self.log = Gsxtlogger(get_pid_log_name(logfile)).get_logger()

        self.source_db = MongDb(mongo_db_target['host'], mongo_db_target['port'], mongo_db_target['db'],
                                mongo_db_target['username'],
                                mongo_db_target['password'], log=self.log)

        self.target_db = MongDb(mongo_db_source['host'], mongo_db_source['port'], mongo_db_source['db'],
                                mongo_db_source['username'],
                                mongo_db_source['password'], log=self.log)

        self.target_db.create_index(self.target_table, [("province", MongDb.ASCENDING)])
        self.target_db.create_index(self.target_table, [("company_name", MongDb.ASCENDING)])

        self.log.info('province = {province} 初始化完成'.format(province=self.province))

    def get_base_info(self, url, item=None):
        return None

    def start_task(self):
        self.log.info('开始清洗 {province} 数据库: source = {source} target = {target}'.format(
            source=self.source_table, target=self.target_table, province=self.province))

        count = 0
        error_count = 0
        for item in self.source_db.traverse(self.source_table, {'is_parse': {'$ne': 1}}):
            try:
                company = item.get('_id', None)
                if company is None:
                    error_count += 1
                    continue

                province = item.get('province', None)
                if province is None:
                    error_count += 1
                    continue

                url = item.get('base_info_url', None)
                if url is None:
                    self.log.error('url 为 None')
                    error_count += 1
                    continue

                base_info = self.get_base_info(url, item)
                if base_info is None:
                    self.log.error('解析url 失败: url = {url}'.format(url=url))
                    error_count += 1
                    continue

                data = dict(_id=company, company_name=company, in_time=util.get_now_time(), base_info=base_info)

                # 存入数据
                self.target_db.save(self.target_table, data)

                # 增加访问标记
                item['is_parse'] = 1
                self.source_db.save(self.source_table, item)
                count += 1
            except Exception as e:
                error_count += 1
                self.log.exception(e)

        self.log.info('count = {count}'.format(count=count))
        self.log.info('error_count = {error}'.format(error=error_count))
